
from transformers import T5Tokenizer,T5ForConditionalGeneration,Trainer,TrainingArguments
from datasets import Dataset,load_dataset
import pandas as pd
import os
os.environ["ACCELERATE_DISABLE_MEMORY"] = "1"
#数据准备
def prepare_prompt_data():
    #示例数据
    data = {
        "input_text":[
            "任务：生成一个关于人工智能的简短介绍",
            "任务：翻译以下内容为英文：机器学习是一种强大的技术",
            "任务：描述机器学习和深度学习的主要区别"
        ],
        "target_text":[
            "人工智能是一种模仿人类智能行为的技术",
            "Machine learning is a powerful technique.",
            "机器学习专注于特征提取，而深度学习通过神经网络自动学习。"
        ]
    }
    df =pd.DataFrame(data)
    return Dataset.from_pandas(df)

#加载数据集
prompt_dataset = prepare_prompt_data()
#划分训练与验证集
train_test_split = prompt_dataset.train_test_split(test_size=0.2)
train_dataset = train_test_split["train"]
val_dataset = train_test_split["test"]
model_name = 't5-small'
tokenizer = T5Tokenizer.from_pretrained(model_name)
model = T5ForConditionalGeneration.from_pretrained(model_name)

#数据处理函数
def preprocess_function(examples):
    inputs = [f"优化提示词:{text}" for text in examples["input_text"]]
    targets = examples["target_text"]
    model_inputs = tokenizer(inputs, return_tensors="pt")
